package com.cf.spider.core.fetch;

import com.cf.auth.model.AuthItem;
import com.cf.auth.model.service.AuthService;
import com.cf.ongl.OnglService;
import com.cf.proxy.model.Proxy;
import com.cf.proxy.model.service.ProxyService;
import com.cf.spider.core.annotation.PageType;
import com.cf.spider.core.annotation.ValueType;
import com.cf.spider.core.fetch.meta.JobItem;
import com.cf.spider.core.model.JobInfo;
import com.cf.spider.core.model.JobTask;
import com.cf.spider.core.model.service.JobTaskService;
import com.cf.util.ObjectJsonUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.transaction.annotation.Transactional;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * boot-awm-FetchService
 *
 * @author <a href="2428922347@qq.com">Zhu mingming</a>
 * @info
 * @since 2018/7/11 上午 10:37
 */
@Slf4j
public class FetchService {

    @Autowired
    private AuthService authService;

    @Autowired
    private ProxyService proxyService;

    @Autowired
    private JobTaskService jobTaskService;

    @Autowired
    private StringRedisTemplate redisTemplate;

    @Autowired
    private OnglService onglService;

    @Transactional
    public void fetch(JobInfo jobInfo, JobTask jobTask) {
        log.info("开始抓取任务【{}】任务是【{}】", jobInfo, jobTask);

        if (StringUtils.isEmpty(jobInfo.getConfig()) || StringUtils.isEmpty(jobTask.getUrl())) {
            jobTaskService.end(jobTask);
        }

        Connection connection = configSpider(jobInfo, jobTask);
        String config = jobInfo.getConfig();


        try {
            Map map = new HashMap();
            String pageData = connection.ignoreContentType(true).execute().body();
            List<JobItem> jobItems = ObjectJsonUtil.toListObject(config, JobItem.class);
            for (JobItem jobItem : jobItems) {
                extract(pageData, jobItem, map);
            }
            String extractDataString = ObjectJsonUtil.toString(map);
            redisTemplate.convertAndSend(jobInfo.getTopic(), extractDataString);
            jobTaskService.success(jobTask, extractDataString);
        } catch (Exception e) {
            log.error("抓取数据失败！【{}】【{}】", jobInfo, jobTask);
            jobTaskService.error(jobTask, e.getMessage());
            return;
        }
    }

    private void extract(String pageData, JobItem jobItem, Map map) {
        if (PageType.HTML.equals(jobItem.getPageType())) {
            Elements selectNode = Jsoup.parse(pageData).select(jobItem.getSelect());
            if (ValueType.HTML.equals(jobItem.getValueType())) {
                map.put(jobItem.getName(), selectNode.html());
            }
            if (ValueType.TEXT.equals(jobItem.getValueType())) {
                map.put(jobItem.getName(), selectNode.text());
            }
            if (ValueType.VALUE.equals(jobItem.getValueType())) {
                map.put(jobItem.getName(), selectNode.val());
            }
        }

        if (PageType.JSON.equals(jobItem.getPageType())) {
            Map dataMap = ObjectJsonUtil.toObject(pageData, Map.class);
            try {
                map.put(jobItem.getName(), onglService.getData(dataMap, jobItem.getSelect()));
            } catch (Exception e) {
                map.put(jobItem.getName(), e.getMessage());
            }
        }
    }

    private Connection configSpider(JobInfo jobInfo, JobTask jobTask) {
        Connection connect = Jsoup.connect(jobTask.getUrl());

        Proxy proxy = proxyService.getProxy();
        if (proxy != null) {
            connect = connect.proxy(proxy.getIp(), proxy.getPort());
        }
        AuthItem authItem = authService.getAuth(jobInfo.getAuthId());
        if (authItem != null) {
            connect = configAuth(connect, authItem);
        }
        connect.userAgent(UserAgentUtil.getUserAgent());
        connect.method(Connection.Method.GET);
        return connect;
    }

    /**
     * 配置
     *
     * @param connect
     * @param authItem
     * @return
     */
    private Connection configAuth(Connection connect, AuthItem authItem) {
        String cookie = authItem.getCookie();
        Map<String, String> cookieMap = keyValue(cookie);
        if (cookieMap.size() > 0) {
            cookieMap.entrySet().forEach(stringStringEntry -> {
                connect.cookie(stringStringEntry.getKey(), stringStringEntry.getValue());
            });
        }
        Map<String, String> headMap = keyValue(authItem.getHead());
        if (headMap.size() > 0) {
            headMap.entrySet().forEach(headEntry -> {
                connect.header(headEntry.getKey(), headEntry.getValue());
            });
        }

        Map<String, String> tokenMap = keyValue(authItem.getToken());
        if (tokenMap.size() > 0) {
            tokenMap.entrySet().forEach(headEntry -> {
                connect.data(headEntry.getKey(), headEntry.getValue());
            });
        }
        return connect;
    }

    public Map<String, String> keyValue(String data) {
        Map map = new HashMap();
        if (StringUtils.isNotEmpty(data)) {
            String[] dataItems = data.split(",");
            if (dataItems.length > 0) {
                for (String dataItem : dataItems) {
                    String[] dataItemData = dataItem.split("=");
                    if (dataItemData != null && dataItemData.length == 2) {
                        map.put(dataItemData[0], dataItemData[1]);
                    }
                }
            }
        }
        return map;
    }


    public void push(JobInfo jobInfo, JobTask task) {
        String data = task.getData();
        redisTemplate.convertAndSend(jobInfo.getTopic(), data);
    }
    
}
